* doQuickOperations() lets us do things like purge thumbnails as fast as possible.
Stat calls, SHA1 checks, and RTTs in general are avoided. It also avoids the
slowness of lazy population of thumbnail SHA1s in Swift (for those made by rewrite.py).
* Removed supportedOperations() to avoid the extra complexity.
* Made a few variable type and exception documentation cleanups.
* Cleaned up unit test file removal a bit and made some functions private.
Change-Id: I6922368c6af7752a6927d96402519132203108a1
return $this->doOperation( $params, $opts );
}
+ /**
+ * Perform a set of independent file operations on some files.
+ *
+ * This does no locking, nor journaling, and possibly no stat calls.
+ * Any destination files that already exist will be overwritten.
+ * This should *only* be used on non-original files, like cache files.
+ *
+ * Supported operations and their parameters:
+ * a) Create a new file in storage with the contents of a string
+ * array(
+ * 'op' => 'create',
+ * 'dst' => <storage path>,
+ * 'content' => <string of new file contents>
+ * )
+ * b) Copy a file system file into storage
+ * array(
+ * 'op' => 'store',
+ * 'src' => <file system path>,
+ * 'dst' => <storage path>
+ * )
+ * c) Copy a file within storage
+ * array(
+ * 'op' => 'copy',
+ * 'src' => <storage path>,
+ * 'dst' => <storage path>
+ * )
+ * d) Move a file within storage
+ * array(
+ * 'op' => 'move',
+ * 'src' => <storage path>,
+ * 'dst' => <storage path>
+ * )
+ * e) Delete a file within storage
+ * array(
+ * 'op' => 'delete',
+ * 'src' => <storage path>,
+ * 'ignoreMissingSource' => <boolean>
+ * )
+ * f) Do nothing (no-op)
+ * array(
+ * 'op' => 'null',
+ * )
+ *
+ * Boolean flags for operations (operation-specific):
+ * 'ignoreMissingSource' : The operation will simply succeed and do
+ * nothing if the source file does not exist.
+ *
+ * Return value:
+ * This returns a Status, which contains all warnings and fatals that occured
+ * during the operation. The 'failCount', 'successCount', and 'success' members
+ * will reflect each operation attempted for the given files. The status will be
+ * considered "OK" as long as no fatal errors occured.
+ *
+ * @param $ops Array Set of operations to execute
+ * @return Status
+ */
+ final public function doQuickOperations( array $ops ) {
+ if ( $this->isReadOnly() ) {
+ return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
+ }
+ foreach ( $ops as &$op ) {
+ $op['overwrite'] = true; // avoids RTTs in key/value stores
+ }
+ return $this->doQuickOperationsInternal( $ops );
+ }
+
+ /**
+ * @see FileBackend::doQuickOperations()
+ */
+ abstract protected function doQuickOperationsInternal( array $ops );
+
/**
* Concatenate a list of storage files into a single file system file.
* The target path should refer to a file that is already locked or
);
}
+ /**
+ * @see FileBackend::doQuickOperationsInternal()
+ * @return Status
+ */
+ public function doQuickOperationsInternal( array $ops ) {
+ // Do the operations on the master backend; setting Status fields...
+ $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
+ $status = $this->backends[$this->masterIndex]->doQuickOperations( $realOps );
+ // Propagate the operations to the clone backends...
+ foreach ( $this->backends as $index => $backend ) {
+ if ( $index !== $this->masterIndex ) { // not done already
+ $realOps = $this->substOpBatchPaths( $ops, $backend );
+ $status->merge( $backend->doQuickOperations( $realOps ) );
+ }
+ }
+ return $status;
+ }
+
/**
* @see FileBackend::doPrepare()
* @return Status
return $status;
}
+ /**
+ * No-op file operation that does nothing.
+ * Do not call this function from places outside FileBackend and FileOp.
+ *
+ * @param $params Array
+ * @return Status
+ */
+ final public function nullInternal( array $params ) {
+ return Status::newGood();
+ }
+
/**
* @see FileBackend::concatenate()
* @return Status
*/
abstract public function getFileListInternal( $container, $dir, array $params );
- /**
- * Get the list of supported operations and their corresponding FileOp classes.
- *
- * @return Array
- */
- protected function supportedOperations() {
- return array(
- 'store' => 'StoreFileOp',
- 'copy' => 'CopyFileOp',
- 'move' => 'MoveFileOp',
- 'delete' => 'DeleteFileOp',
- 'create' => 'CreateFileOp',
- 'null' => 'NullFileOp'
- );
- }
-
/**
* Return a list of FileOp objects from a list of operations.
* Do not call this function from places outside FileBackend.
* @throws MWException
*/
final public function getOperationsInternal( array $ops ) {
- $supportedOps = $this->supportedOperations();
+ $supportedOps = array(
+ 'store' => 'StoreFileOp',
+ 'copy' => 'CopyFileOp',
+ 'move' => 'MoveFileOp',
+ 'delete' => 'DeleteFileOp',
+ 'create' => 'CreateFileOp',
+ 'null' => 'NullFileOp'
+ );
$performOps = array(); // array of FileOp objects
// Build up ordered array of FileOps...
return $status;
}
+ /**
+ * @see FileBackend::doQuickOperationsInternal()
+ * @return Status
+ * @throws MWException
+ */
+ final protected function doQuickOperationsInternal( array $ops ) {
+ $status = Status::newGood();
+
+ $async = $this->parallelize;
+ $maxConcurrency = $this->concurrency; // throttle
+
+ $statuses = array(); // array of (index => Status)
+ $fileOpHandles = array(); // list of (index => handle) arrays
+ $curFileOpHandles = array(); // current handle batch
+ // Perform the sync-only ops and build up op handles for the async ops...
+ foreach ( $ops as $index => $params ) {
+ $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
+ if ( !MWInit::methodExists( __CLASS__, $method ) ) {
+ throw new MWException( "Operation '{$params['op']}' is not supported." );
+ }
+ $subStatus = $this->$method( array( 'async' => $async ) + $params );
+ if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { // async
+ if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
+ $fileOpHandles[] = $curFileOpHandles; // push this batch
+ $curFileOpHandles = array();
+ }
+ $curFileOpHandles[$index] = $subStatus->value; // keep index
+ } else { // error or completed
+ $statuses[$index] = $subStatus; // keep index
+ }
+ }
+ if ( count( $curFileOpHandles ) ) {
+ $fileOpHandles[] = $curFileOpHandles; // last batch
+ }
+ // Do all the async ops that can be done concurrently...
+ foreach ( $fileOpHandles as $fileHandleBatch ) {
+ $statuses = $statuses + $this->executeOpHandlesInternal( $fileHandleBatch );
+ }
+ // Marshall and merge all the responses...
+ foreach ( $statuses as $index => $subStatus ) {
+ $status->merge( $subStatus );
+ if ( $subStatus->isOK() ) {
+ $status->success[$index] = true;
+ ++$status->successCount;
+ } else {
+ $status->success[$index] = false;
+ ++$status->failCount;
+ }
+ }
+
+ return $status;
+ }
+
/**
* Execute a list of FileBackendStoreOpHandle handles in parallel.
* The resulting Status object fields will correspond
*
* @param $handles Array List of FileBackendStoreOpHandle objects
* @return Array Map of Status objects
+ * @throws MWException
*/
final public function executeOpHandlesInternal( array $fileOpHandles ) {
wfProfileIn( __METHOD__ );
/**
* @see FileBackendStore::executeOpHandlesInternal()
* @return Array List of corresponding Status objects
+ * @throws MWException
*/
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
/**
* Get the cache key for a container
*
- * @param $container Resolved container name
+ * @param $container string Resolved container name
* @return string
*/
private function containerCacheKey( $container ) {
/**
* Set the cached info for a container
*
- * @param $container Resolved container name
+ * @param $container string Resolved container name
* @param $val mixed Information to cache
* @return void
*/
/**
* Delete the cached info for a container
*
- * @param $container Resolved container name
+ * @param $containers string Resolved container name
* @return void
*/
final protected function deleteContainerCache( $container ) {
/**
* Get the cache key for a file path
*
- * @param $path Storage path
+ * @param $path string Storage path
* @return string
*/
private function fileCacheKey( $path ) {
/**
* Set the cached stat info for a file path
*
- * @param $path Storage path
+ * @param $path string Storage path
* @param $val mixed Information to cache
* @return void
*/
/**
* Delete the cached stat info for a file path
*
- * @param $path Storage path
+ * @param $path string Storage path
* @return void
*/
final protected function deleteFileCache( $path ) {
}
// Try to do all the operations concurrently...
$statuses = $statuses + $backend->executeOpHandlesInternal( $opHandles );
- // Marshall and merge all the responses (blocking)...
+ // Marshall and merge all the responses...
foreach ( $performOpsBatch as $i => $fileOp ) {
if ( !$fileOp->failed() ) { // failed => already has Status
$subStatus = $statuses[$i];
class FileBackendTest extends MediaWikiTestCase {
private $backend, $multiBackend;
private $filesToPrune = array();
- private $dirsToPrune = array();
private static $backendToUse;
function setUp() {
return $cases;
}
+ public function testDoQuickOperations() {
+ $this->backend = $this->singleBackend;
+ $this->doTestDoQuickOperations();
+ $this->tearDownFiles();
+
+ $this->backend = $this->multiBackend;
+ $this->doTestDoQuickOperations();
+ $this->tearDownFiles();
+ }
+
+ private function doTestDoQuickOperations() {
+ $backendName = $this->backendClass();
+
+ $base = $this->baseStorePath();
+ $files = array(
+ "$base/unittest-cont1/fileA.a",
+ "$base/unittest-cont1/fileB.a",
+ "$base/unittest-cont1/fileC.a"
+ );
+ $ops = array();
+ $purgeOps = array();
+ foreach ( $files as $path ) {
+ $status = $this->prepare( array( 'dir' => dirname( $path ) ) );
+ $this->assertGoodStatus( $status,
+ "Preparing $path succeeded without warnings ($backendName)." );
+ $ops[] = array( 'op' => 'create', 'dst' => $path, 'content' => mt_rand(0,50000) );
+ $purgeOps[] = array( 'op' => 'delete', 'src' => $path );
+ }
+ $purgeOps[] = array( 'op' => 'null' );
+ $status = $this->backend->doQuickOperations( $ops );
+ $this->assertGoodStatus( $status,
+ "Creation of source files succeeded ($backendName)." );
+
+ foreach ( $files as $file ) {
+ $this->assertTrue( $this->backend->fileExists( array( 'src' => $file ) ),
+ "File $file exists." );
+ }
+
+ $status = $this->backend->doQuickOperations( $purgeOps );
+ $this->assertGoodStatus( $status,
+ "Quick deletion of source files succeeded ($backendName)." );
+
+ foreach ( $files as $file ) {
+ $this->assertFalse( $this->backend->fileExists( array( 'src' => $file ) ),
+ "File $file purged." );
+ }
+ }
+
/**
* @dataProvider provider_testConcatenate
*/
$this->tearDownFiles();
}
- function doTestRecursiveClean() {
+ private function doTestRecursiveClean() {
$backendName = $this->backendClass();
$base = $this->baseStorePath();
}
// concurrency orientated
- function doTestDoOperations2() {
+ private function doTestDoOperations2() {
$base = $this->baseStorePath();
$fileAContents = '3tqtmoeatmn4wg4qe-mg3qt3 tq';
"Correct file SHA-1 of $fileC" );
}
- function doTestDoOperationsFailing() {
+ private function doTestDoOperationsFailing() {
$base = $this->baseStorePath();
$fileA = "$base/unittest-cont2/a/b/fileA.txt";
// test helper wrapper for backend prepare() function
private function prepare( array $params ) {
- $this->dirsToPrune[] = $params['dir'];
return $this->backend->prepare( $params );
}
foreach ( $containers as $container ) {
$this->deleteFiles( $container );
}
- foreach ( $this->dirsToPrune as $dir ) {
- $this->recursiveClean( $dir );
- }
- $this->filesToPrune = $this->dirsToPrune = array();
+ $this->filesToPrune = array();
}
private function deleteFiles( $container ) {
array( 'force' => 1, 'nonLocking' => 1 ) );
}
}
- }
-
- private function recursiveClean( $dir ) {
- $this->backend->clean( array( 'dir' => $dir, 'recursive' => 1 ) );
+ $this->backend->clean( array( 'dir' => "$base/$container", 'recursive' => 1 ) );
}
function assertGoodStatus( $status, $msg ) {
dépôts / lhc / web / wiklou.git / commitdiff